# -*- coding:utf-8 -*-
import urllib2,threading,random,time,sys,traceback,datetime

from spider import Spider
from pipeline import Pipeline
from http.request import Request
from http.response import Response
from items import Item
import funcs
import settings


class Downloader(threading.Thread):
    def __init__(self,thread_id,scheduler,engine):
        threading.Thread.__init__(self)
        self.thread_id=thread_id
        self.scheduler=scheduler
        self.engine=engine
        
        self.cxn=funcs.get_db_cxn()
        self.cur=self.cxn.cursor()
        
        self.spider=Spider()
        self.pipeline=Pipeline(self.cur,self.cxn,self.thread_id)
        
    def run(self):
        '''运行，不断循环，直到请求获取为空'''
        self.is_running=True
        try_times=0
        while self.engine.is_running==True and try_times<=50:
            try:
                request=self.scheduler.get_request(self.cur,self.cxn)
                if isinstance(request,Request):
                    response=self.download(request)
                    if response.body=='':
                        #self.engine.logger.error('thread'+str(self.thread_id)+' download fail : '+response.url)
                        pass
                    else:
                        self.start_spider(response)
                    try_times=0
                elif isinstance(request,str):
                    print '====get_request: '+request
                    try_times=0
                else:
                    print '----download.run: '+'thread '+str(self.thread_id)+' url_queue is empty!'
                    try_times+=1
                    time.sleep(2)
            except Exception:
                etype,value,tb = sys.exc_info()
                msg=traceback.format_exception(etype, value, tb)
                msg.insert(0,'thread'+str(self.thread_id)+':\n')
                self.engine.logger.error(''.join(msg))
                print '####download.run: '+str(msg)
                for lock in self.engine.d_locks:
                    try:
                        self.engine.d_locks[lock].release()
                    except Exception:
                        sys.exc_info()
                time.sleep(1)
        self.thread_end()
        
    def thread_end(self):
        self.is_running=False
        if self.cur:
            self.cur.close()
        if self.cxn:
            self.cxn.close()
        print 'THREAD %d STOP' %self.thread_id
        
    def download(self,request,try_times=8):
        '''下载网页内容'''
        try:
            if try_times==8:
                pass
            else:
                time.sleep(try_times**2)
            try_times+=1
            user_agents = [                
                'Baiduspider+(+http://www.baidu.com/search/spider.htm)',
                #'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
                #'msnbot-media/1.0 (+http://search.msn.com/msnbot.htm)',
                #'Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/;)'
                #'Mozilla/5.0 (compatible; iaskspider/1.0; MSIE 6.0)',
                ]
            headers={
                'User-Agent':user_agents[random.randint(0,len(user_agents)-1)],
                #'User-Agent':'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1',
                #'Accept-Language':'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3',
                'Referer':settings.S_start_urls[0]
                }
            req=urllib2.Request(url=request.url,headers=headers)
            res=urllib2.urlopen(req,timeout=settings.S_timeout)
            if request.url.strip() == res.url.strip():                      #不保存重定向的网页
                data=res.read()
            else:
                data=''
                print 'RRRR %s is a redirect page'%request.url
                time.sleep(1)
            res.close()
        except urllib2.HTTPError,e:
            data=''
            print str(datetime.datetime.now())
            err_msg='|'.join((str(self.thread_id),str(e),request.url))
            print 'XXXXdownload.download: '+str(err_msg)
            if e.code!=404 and e.code!=500:
                self.engine.logger.error(err_msg)
            if e.code==403 and try_times<=10:
                funcs.login_website()
                resp=self.download(Request(request.url),try_times)
                return resp
            time.sleep(1)
        except urllib2.URLError,e:
            data=''
            err_msg='download.download: '+'thread '+str(self.thread_id)+':'+str(e)
            self.engine.logger.error(err_msg)
            time.sleep(1)
        response=Response(request.url,body=data,request=request)
        return response
    
    def start_spider(self,response):
        '''开始分析网页内容，提取链接'''
        parse=self.spider.parse(response)
        try:
            while True:
                parse_result=parse.next()
                if isinstance(parse_result,Item):
                    self.pipeline.pipeline(parse_result)
                elif isinstance(parse_result,Request):
                    self.scheduler.add_request(parse_result,self.cur,self.cxn)
        except StopIteration:
            pass












